import sys
sys.path.append('/usr/local/lib/python2.7/dist-packages')
import os
import math
import EmilyTreeNode
import urllib
import urllib2
import feedparser
import HTMLParser
import re
#import svgwrite
import codecs
import datetime
import calendar
import multiprocessing
import pymongo
import pymongo.bson
import pymongo.mongo_client
import pymongo.database
import pymongo.collections
import pymongo.operations

client=pymongo.mongo_client.MongoClient(host=os.environ['OPENSHIFT_MONGODB_DB_HOST'],
                                        port=os.environ['OPENSHIFT_MONGODB_DB_PORT'])
db=pymongo.database.Database(client,'emily')
db.authenticate(os.environ['OPENSHIFT_MONGODB_DB_USERNAME'],
                password=os.environ['OPENSHIFT_MONGODB_DB_PASSWORD'])
blogs=db.blogs
links=db.links
recommendations=db.recommendations
blogs.create_index('topic')
links.create_index('blogs')
recommendations.create_index([('date',pymongo.DESCENDING)])

SentenceEnd=re.compile(u"""[.?!]['"]*\s+""")
StripXML=re.compile(u'<[^>]*>')
SplitWords=re.compile(u"""[.?!,;:"]*\s+""")
RelFinder=re.compile(u'(?<=rel=").*?(?=")')
LinkFinder=re.compile(u'(?<=<).*?(?=>)')

def parallel(method):
    """Decorator for parallel processes"""
    def call(*args,**kwargs):
        """Runs the method in a process"""
        p=multiprocessing.Process(target=method,args=args,kwargs=kwargs)
        p.start()
    return call

def convert_date(time_tuple):
    """Converts a time_tuple to a datetime"""
    return datetime.datetime.fromtimestamp(calendar.timegm(time_tuple))

def ParseLinkHeader(header):
    """Extracts links and relationships from a html link header"""
    return dict(zip(RelFinder.findall(header),LinkFinder.findall(header)))

def Blogs(query):
    """Iterates over all blogs that satisfy a query"""
    for blog in blogs.find(query):
        yield EmilyBlogModel.from_bson(blog)

def GetBlog(query):
    """Finds one blog that satisfies the query"""
    return EmilyBlogModel.from_bson(blogs.find_one(query))

def Recommendations(links):
    """Gets recommendation entries given their ids"""
    for rec in recommendations.find({'_id':{'$in':links}}).order({'date':pymongo.DESCENDING}):
        yield rec


def Blog_is_New(url):
    """True if blog is not already in system"""
    return blogs.count({'_id':url})==0

class EmilyHTMLParser(HTMLParser.HTMLParser,object):
    """Class to extract <title> and <link rel="alternate"> from blog"""
    def __init__(self):
        """Two custom variables to contain title and FeedURL"""
        super(EmilyHTMLParser,self).__init__()
        self.title=''
        self.FeedURL=None
        self.InTitle=False

    def handle_starttag(self,tag,attrs):
        """Detects the <title> tag"""
        if tag=='title':
            self.InTitle=True

    def handle_endtag(self,tag):
        """Detects the </title> tag"""
        if tag=='title':
            self.InTitle=False

    def handle_data(self,data):
        """Extracts the blog title"""
        if self.InTitle:
            self.title+=data

    def handle_startendtag(self,tag,attrs):
        """Finds the feed url"""
        if tag=='link':
            AttrDict=dict(attrs)
            if AttrDict['rel']=='alternate':
                if self.FeedURL==None or ('type' in AttrDict and AttrDict['type']=="application/atom+xml"):
                    self.FeedURL=AttrDict['href']
            

parser=EmilyHTMLParser()



@parallel
def SetupLinks(blog):
    """Sets up links between blog and the blogs already in the system"""
    BlogLinks=[{'blogs':[blog.url,other.url],
                'strength':blog.Similarity(other)}
               for other in Blogs({'_id':{'$ne':blog.url}})]
    links.insert_many(BlogLinks)
    

##def InitLinks():
##    """Sets up links for all blogs in the system"""
##    @parallel
##    def LinkFinder(blog):
##        blog.ready=False
##        blog.blog=EmilyBlogModel(blog.url)
##        blog.Update(feed=None,callback=SubscribeCallback(blog))
##    q=EmilyBlogModelAppEngineWrapper.query().map_async(LinkFinder)
##    q.get_result()

class EmilyBlogModel(object):
    """Model of the semantic structure of a blog"""
    @classmethod
    def from_bson(cls,Bson):
        """For retrieving from mongodb"""
        result=cls()
        result.url=Bson._id
        result.topic=Bson.topic
        result.hub=Bson.hub
        result.title=Bson.title
        result.H=Bson.H
        result.words={word:EmilyTreeNode.from_bson(node)
                      for (word,node) in Bson.words.iteritems()}
        result.recommendations=set(Bson.recommendations)
        result.Tree=EmilyTreeNode.from_bson(Bson.Tree)
        return result
    
    def __init__(self,url==None):
        """Sets up a model of the blog at url"""
        self.words={}
        self.recommendations=set()
        self.Tree=None
        self.H=0
        self.N=0
        self.url=url
        self.updated=datetime.datetime.now()
        if self.url is not None:
            for line in urllib2.urlopen(url):
                parser.feed(codecs.decode(line,'utf-8'))
            self.title=parser.title
            #Feed=feedparser.parse(urllib2.urlopen(parser.FeedURL))
            #self.Update(Feed)
            self.hub=None
            self.topic=parser.FeedURL
            Feed=feedparser.parse(urllib2.urlopen(self.topic))
            for link in Feed.feed.links:
                if link.rel=='hub':
                    self.hub=link.href
                if link.rel=='self':
                    self.topic=link.href

    def bson(self):
        """Returns bson to save in mongodb"""
        return pymongo.bson.son([('_id',self.url),
                                 ('topic',self.topic),
                                 ('title',self.title),
                                 ('hub',self.hub),
                                 ('updated',self.updated),
                                 ('H',self.H),
                                 ('N',self.N),
                                 ('words',pymongo.bson.son([(word,node.bson())
                                                            for node in self.words.iteritems()])),
                                 ('recommendations',list(self.recommendations)),
                                 ('Tree',self.Tree.bson())])

    def subscribe(self):
        """Sends a subscription request to the hub"""
        req=urllib2.Request(self.hub,urllib.urlencode({'hub.callback':"https://emily-petebleackley.rhcloud.com/update",
                                                       'hub.mode':'subscribe',
                                                       'hub.topic':self.topic,
                                                       'hub.verify':'async'}))
        urllib2.urlopen(req)
        
        
        
        

    def Similarity(self,other):
        """Similarity metric for two blogs. An entropy-weighted variation
           on a Tanimoto metric.
           Sum the entropy-weighted similarity of the environments of all
           words that occur in both blogs, and divide by the total entropy of
           the two blogs"""
        result=0
        for word in self.Tree:
            if word in other.Tree:
                result+=self[word].Similarity(other[word])
        return result/(self.H+other.H)

   
    @parallel
    def Update(self,feed=None):
        """Takes data from a feedparser feed and adds it to the model"""
        New=False
        if feed is None:
            feed=feedparser.parse(urllib2.urlopen(self.topic))
            New=True
        rawdata=u'\n'.join((u'\n'.join((x.value for x in entry.content))
                            for entry in feed.entries))
        Sentences=[set(SplitWords.split(sentence))
                   for sentence in SentenceEnd.split(StripXML.sub('',parser.unescape(rawdata)))]
        self.UpdateTree(Sentences)
        blogs.replace_one({'_id':self.url},self.bson(),True)
        if New:
            self.subscribe()
        else:
            self.UpdateLinks(feed)
        
    def UpdateTree(self,Sentences):
        """Updates the tree structure with new Sentences"""
        deltaN=len(Sentences)
        self.N+=deltaN
        KnownWords=set()
        for sentence in Sentences:
            for word in sentence:
                if word!=u'' and not word.isspace() and word not in KnownWords:
                    known=word in self.words
                    FoundAt=set((i for (i,s) in enumerate(Sentences)
                                                             if word in s))
                    self.words.setdefault(word,
                                          EmilyTreeNode.EmilyTreeNode(set([word]),
                                                                      FoundAt,
                                                                      self.N))
                    if known:
                        self.words[word].Update(FoundAt,deltaN)
                    KnownWords.add(word)
        self.GrowTree()
    
    @parallel
    def UpdateLinks(self,feed):
        """Updates clustering and recommendation data"""
        entries=[{'title':entry.title,
                  'blogtitle':self.title,
                  'summary':entry.summary,
                  'permalink':entry.link,
                  'blogurl':self.url,
                  '_id':entry.id,
                  'date':convert_date(entry.published_parsed)}
                 for entry in feed.entries]
        recommendations.insert_many(entries)
        updated_links=[]
        updated_blogs=[]
        for link in links.find({'blogs',self.url}):
            other_url=link.blogs[0] if link.blogs[1]==self.url else link.blogs[0]
            other=GetBlog(other_url)
            sim=self.Similarity(other)
            if sim>link.strength:
                for entry in entries:
                    other.Recommend(entry._id)
                other.updated=datetime.datetime.now()
                updated_blogs.append(other)
            updated_links.append(pymongo.operations.UpdateOne({'_id':link._id},
                                                              {'$set':{'strength':sim}}))
        blogs.BulkWrite([pymongo.operations.UpdateOne({'_id':blog.url},
                                                      {'$addToSet':{'recommendations':{'$each':list(blog.recommendations)}},
                                                       '$set':{'updated':blog.updated}})
                         for blog in updated_blogs],False)
        links.BulkWrite(updated_links,False)
        for blog in updated_blogs:
            blog.Notify()
                                 
        
                                
                                
                  
        

    def GrowTree(self):
        """Creates a tree structure representing the semantic relationships
           between the words in the blog"""
        Similarities=[]
        for word in self.words:
            if len(self.words[word].sentences)>1:
                node={'node':self.words[word],
                      'score':float('-inf'),
                      'neighbour':None}
                for (i,node2) in enumerate(Similarities):
                    Sim=node['node'].Entropy(node2['node'])
                    if Sim>node['score']:
                        node['score']=Sim
                        node['neighbour']=node2['node']
                    if Sim>node2['score']:
                        Similarities[i]['score']=Sim
                        Similarities[i]['neighbour']=node['node']
                Similarities.append(node)
        while len(Similarities)>1:
            old=len(Similarities)
            Similarities.sort(key=lambda x:x['score'])
            a=Similarities.pop()
            node={'node':a['node']+a['neighbour'],
                  'score':float('-inf'),
                  'neighbour':None}
            affected=[{'node':x['node'],
                       'score':float('-inf'),
                       'neighbour':None} for x in Similarities
                      if x['node']!=a['neighbour'] and (x['neighbour']==a['node']
                                                        or x['neighbour']==a['neighbour'])]
            Similarities=[x for x in Similarities
                          if x['node']!=a['neighbour']
                          and x['neighbour']!=a['node']
                          and x['neighbour']!=a['neighbour']]
            affected.append(node)
            for node in affected:         
                for (i,node2) in enumerate(Similarities):
                    Sim=node['node'].Entropy(node2['node'])
                    if Sim>node['score']:
                        node['score']=Sim
                        node['neighbour']=node2['node']
                    if Sim>node2['score'] or node2['neighbour']==a['node'] or node2['neighbour']==a['neighbour']:
                        Similarities[i]['score']=Sim
                        Similarities[i]['neighbour']=node['node']
                Similarities.append(node)
            assert len(Similarities)<old
        self.Tree=Similarities[0]['node']
        self.H=self.Tree.TotalEntropy

    def WordGraph(self):
        """Returns the most significant words in the blog in a format suitable
           to be plotted on a d3.js force-directed graph"""
        result={'nodes':[{'word':word,
                          'H':self.Tree[word].TotalEntropy()}
                         for word in self.Tree
                         if self.Tree[word].TotalEntropy()>1],
                'links':[]}
        for (i,node) in enumerate(result['nodes'][:-1]):
            word1=node['word']
            Node1=self.Tree[word1]
            for j in range(i+1,len(result['nodes'])):
                word2=result['nodes'][j]
                Node2=self.Tree[word2]
                result['links'].append({'source':i,
                                        'target':j,
                                        'strength':self.Tree.LinkEntropy(word1,word2)})
        return result

    def Search(self,words):
        """Returns the entropy associated with the deepest node in the tree that
           contains all words"""
        return self.Tree.search(words)

##    def WordCloud(self):
##        """Returns a wordcloud as SVG"""
##        
##        cloud,size=self.Tree.NodePositions()
##        svg=svgwrite.Drawing(size=size)
##        svg.add(cloud)
##        return svg.tostring()

    def Notify(self):
        """Notifies a pubsubhubbub hub when the blog's recommendations
           have been updated"""
        data={'hub.mode':'publish',
              'hub.url':'https://emily-petebleackley.rhcloud.com/recommend?url={}'.format(urllib.quote_plus(self.url))}
        urllib2.urlopen('https://pubsubhubbub.appspot.com',urllib.urlencode(data))
        
        

    def Recommend(self,permalink):
        """Add a recommendation for this blog"""
        self.recommendations.add(permalink)

##    @ndb.tasklet
##    @classmethod
##    def PruneLinks(cls):
##        """Removes weak links"""
##        EmilyLink.query(EmilyLink.strength<EmilyBlogModel.Threshold).delete_async()
                                       

##class EmilyCluster(ndb.model):
##    """Represents a group of similar blogs"""
##     blogs=ndb.StringProperty(repeated=True)
##     graph=ndb.PickleProperty()

     #@ndb.tasklet
     #def FindBestMatch(self,blog):
        
##if __name__=="__main__":
##    emily=EmilyBlogModel("http://fantasticaldevices.blogspot.com")
##    with codecs.open("fantasticaldevices.svg",'w','utf-8') as svg:
##        svg.write(emily.WordCloud())
                

    

    
        
